package com.proxyhub.crawler;

import com.google.common.collect.ImmutableList;
import com.proxyhub.annotation.Crawler;
import com.proxyhub.model.Proxy;
import com.proxyhub.utils.CommonUtil;
import com.proxyhub.utils.ProxyUtil;
import com.proxyhub.utils.RequestUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Service;

import java.util.ArrayList;
import java.util.List;

/**
 * 年少代理抓取
 * Created by linxsh on 2017/7/7.
 */
@Crawler("年少")
@Service
public class NianshaoCrawler extends BaseCrawler {
    List<String> seedUrls = ImmutableList.of(
            "http://www.nianshao.me/?stype=1",
            "http://www.nianshao.me/?stype=2"
    );

    @Override
    public void crawl() {
        for(String seedUrl: seedUrls) {
            for(int i = 1; i <= 100; ++i) {
                Document dataPage = RequestUtil.doGetDoc(seedUrl + i, CommonUtil.GBK);
                Elements trs = dataPage.select("table.table>tbody>tr");
                List<Proxy> proxies = new ArrayList<Proxy>();
                for(Element tr: trs) {
                    String host = tr.select("td:eq(0)").text();
                    String port = tr.select("td:eq(1)").text();
                    String type = tr.select("td:eq(3)").text();
                    boolean isHttps = tr.select("td:eq(4)").text().equalsIgnoreCase("https");
                    String location = tr.select("td:eq(2)").text().trim();
//                    String delay = tr.select("td:eq(5)").text().replaceAll("[^\\.\\d]", "");

                    if(!CommonUtil.isIP(host)) {
                        continue;
                    }

                    if(StringUtils.isBlank(port) || !StringUtils.isNumeric(port)) {
                        continue;
                    }

                    boolean valid = ProxyUtil.validateProxy(host, NumberUtils.toInt(port));
                    if(valid) {
                        proxies.add(new Proxy(host, NumberUtils.toInt(port), type, location, isHttps, -1, valid));
                    }
                }
                proxyRepo.insertIfNotExist(proxies);
            }
        }
    }
}
